from urllib2 import urlopen
from urllib2 import HTTPError
from urllib2 import URLError
from bs4 import BeautifulSoup
from urlparse import urljoin
from urlparse import urldefrag


def url_format(url):
    url = urldefrag(url)[0]
    if "?icn=" in url:
        return url[0:url.find("?icn=")]
    # print url
    # url = url[len(config.base_url):len(url)]
    # print url
    return url


def get_html(url):
    try:
        html = urlopen(url)
        html = html.read()
    except HTTPError as e:
        return e
    except URLError as e:
        return e
    return html


def get_urls(url, html):
    result = set()
    bs_obj = BeautifulSoup(html, 'lxml')
    for link in bs_obj.findAll("a"):
        if 'href' in link.attrs:
            href = link.attrs['href']
            href = urljoin(url, href)
            if href not in result:
                # print href
                href = url_format(href)
                # print href
                result.add(href)
    return result


def main():
    url = 'https://streeteasy.com/for-sale/soho?page=1&sort_by=price_desc'
    print get_html(url)


if __name__ == '__main__':
    main()
